# -*- coding: utf-8 -*-
import MySQLdb

import scrapy
import datetime
import time
from scrapy.http import Request
from urllib import parse

from horsveision.items import CsrcItem


class CsrcSpider(scrapy.Spider):
    name = 'csrc'
    allowed_domains = ['www.csrc.gov.cn/pub/newsite']
    start_urls = ['http://www.csrc.gov.cn/pub/newsite/zjhxwfb/xwdd/']
    db = MySQLdb.connect('localhost', 'root', '698a80e4212ba38f', 'toujiao', charset='utf8', use_unicode=True)
    # db = MySQLdb.connect('192.168.100.101', 'root', 'root', 'haohsi', charset='utf8', use_unicode=True)

    def parse(self, response):
        content_nodes = response.css('#myul li')
        now_date = datetime.datetime.now().strftime('%Y-%m-%d')
        now_time = time.mktime(time.strptime(now_date, '%Y-%m-%d'))
        for content_node in content_nodes:
            reslease_date = content_node.css('span::text').extract_first('')
            reslease_time = time.mktime(time.strptime(reslease_date, '%Y-%m-%d'))
            if int(reslease_time) == int(now_time):
                title = content_node.css('a::attr(title)').extract_first('')
                with self.db.cursor() as cursor:
                    sql = "SELECT title FROM spider_content WHERE title='%s'" % title
                    cursor.execute(sql)
                    result = cursor.fetchone()
                    if result is None:
                        content_url = content_node.css('a::attr(href)').extract_first('')
                        yield Request(url=parse.urljoin(response.url, content_url), callback=self.parse_detail,
                                      dont_filter=True)

    def parse_detail(self, response):
        # 提取文章的具体内容
        csrc_item = CsrcItem()
        title = response.css('.content .title::text').extract()[0]
        send_time = response.css('.in_main .time span::text').extract()[1].replace('时间：', '').strip()
        source = response.css('.in_main .time span::text').extract()[2].replace('来源：', '').strip()
        content = response.xpath("//div[@class='content']/div[3]").extract()[0]

        csrc_item['title'] = title
        csrc_item['send_time'] = int(time.mktime(time.strptime(send_time, '%Y-%m-%d')))
        csrc_item['source'] = source
        csrc_item['content'] = content
        csrc_item['cover_img'] = ''

        yield csrc_item
